#!/usr/bin/python
import os
import re
import sys

# download files, and output filepaths
# if file exists in source_dir/, then do not download
# write a 'file_path' list into output_filename

if len(sys.argv)!=4: 
	print '''Arguments: urlfile source_dir output_filename
urlfile - a file which consists of urls
source_dir - local directory. downloaded files are saved to source_dir/
'''
	raise Exception('Require three argument.')

urlfile=sys.argv[1]
source_dir=sys.argv[2]
output_filename=sys.argv[3]

if os.path.exists(urlfile)==False: raise Exception('%s does not exist.'%urlfile)
if os.path.exists(source_dir)==False: raise Exception('%s does not exist.'%source_dir)
if os.path.exists(output_filename): raise Exception('%s exists. I will not overwrite it.'%output_filename)

out=open(output_filename,'w')

os.chdir(os.path.dirname(os.path.abspath(sys.argv[0]))) # chdir to the directory where this script stays
urlfile=open(urlfile,'r')
for line in urlfile:
	line = line.strip()
	if len(line)==0: # empty line
		continue
	elif line[0]=='#': # is a comment
		continue
	elif line.find('http://')==0 or line.find('ftp://')==0: # url
		filename=re.search('[^/]+$',line)
		if filename==None: # file name not found
			raise Exception('File `WhereAreThey` format error.')
		filename=filename.group(0)
		path=os.path.join(source_dir,filename)
		if os.path.exists(path)==False:
			cmd='wget -c --tries=20 %s'%line # continue getting, 20 tries, 
				# turn off verbose without being completely quiet
			if os.system(cmd): 
				raise Exception('Cannot download %s'%filename, cmd)
			if os.system('mv %s %s'%(filename,source_dir)):
				raise Exception('Cannot move %s into %s'%(filename, source_dir))
		out.write(path+'\n') # output filepath
	else:
		raise Exception('Cannot understand line %s'%line)
